package wordsDistribution;

import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.InvalidFormatException;

import com.google.gdata.util.common.html.HtmlToText;

public class WordsDistribution {

	public static final String SEPARATOR = " ";

	/**
	 * @param args
	 * @throws IOException 
	 * @throws FileNotFoundException 
	 * @throws InvalidFormatException 
	 */
	public static void main(String[] args) throws InvalidFormatException, FileNotFoundException, IOException {
		
		TokenizerME tokenizer = new TokenizerME(new TokenizerModel(new FileInputStream("en-token.bin")));
		
		String dataFolder = "/home/pjbarrio/Documents/RelationExtractionStuff/DatabasesInterfaces/FinalSelection/Data/";
		
		String distribution = "/home/pjbarrio/Documents/RelationExtractionStuff/DatabasesInterfaces/WordsDistribution/";

		File folder = new File(dataFolder);
		
		File[] files = folder.listFiles();
		
		Arrays.sort(files);
		
		for (File file : files) {
			
			Map<String,Integer> map = new HashMap<String, Integer>();
			
			String plainText = HtmlToText.htmlToPlainText(readFile(file));
			
			String[] tokens = tokenizer.tokenize(plainText);
			
			for (String token : tokens) {
				
				updateMap(token,map);
				
			}
			
			save(map,new File(distribution + file.getName().replace(".html", "")));

			System.out.println(file.getName());
			
		}
		
	}

	private static void save(Map<String, Integer> map, File file) throws IOException {
		
		BufferedWriter bw = new BufferedWriter(new FileWriter(file));
		
		for (Entry<String, Integer> entry : map.entrySet()) {
			
			bw.write(entry.getKey() + SEPARATOR + entry.getValue());
			
			bw.newLine();
			
		}
		
		bw.close();
	}

	private static void updateMap(String token, Map<String, Integer> map) {
		
		Integer freq = map.get(token);
		
		if (freq == null){
			freq = 0;
		}
		
		map.put(token, freq+1);
	}

	private static String readFile(File file) {
		
	    byte[] buffer = new byte[(int) file.length()];
	    BufferedInputStream f = null;
	    try {
	        f = new BufferedInputStream(new FileInputStream(file.getAbsolutePath()));
	        f.read(buffer);
	    } catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} finally {
	        if (f != null) try { f.close(); } catch (IOException ignored) { }
	    }
	    return new String(buffer);
		
	}
	
}
